<?php
namespace Home\Controller;
use Think\Controller;
use Model\Adminuser;
class IndexController extends Controller {
    public $city_list = 'http://www.dianping.com/citylist';
    public function _initialize(){
        header("Content-type: text/html; charset=utf-8");
        set_time_limit(0);
        // vendor('Snoopy.Snoopy');
    }
    public function index(){
    	echo '抓取,只是简单的一按~<br>';
        $this->display();

    }
    /**
     * [\x80-\xff] gbk
     * [\x{4e00}-\x{9fa5}]+  utf8
     * 抓取城市列表
     */
    public function pull_city_list(){
        $list = M('country')->field('*,country name')->where('pid=0')->select();
        echo M('citylist')->Addall($list);
        dump($list);
    }
    /**
     * 获取二级行政区 列表
     * @return [type] [description]
     */
    public function pull_second_city(){
        $list = M('country')->field('*,country name,pid citylist_id')->where('pid>0 && pid<35')->select();
        echo M('second_citylist')->Addall($list);
        dump($list);
    }

    /**
     * 获取三级级行政区 列表 (改成热门城市查询3级地址)
     * @return [type] [description]
     */
    public function pull_third_city(){
        $list = M('country')->field('*,country name,pid second_citylist_id')->where('pid>34')->select();
        echo M('third_citylist')->Addall($list);
        dump($list);
    }

    // public function pull_third_city(){
    //     set_time_limit(0);
    //     $map['c.name'] = array('in',array('北京','上海','广州','深圳','天津','杭州','南京','苏州','成都','武汉','重庆','西安',));
    //     $citylist = M('second_citylist')->field('s.*')->alias('s')->join('left join citylist c ON c.id=s.citylist_id')->where('s.is_pull=0')->where($map)->select();
    //     foreach ($citylist as $key => $value) {
    //         $content_parent = $this->pull_url('http://'.$value['url']);
    //         if( preg_match("/抱歉！页面无法访问/", $content_parent) ) {
    //             echo $value['name'].$value['url']."获取失败";
    //             die;
    //         }
    //         if(empty($content_parent)) {
    //             echo $value['name'].$value['url']."页面抓取为空字符串,或需要填写验证码";
    //             file_put_contents('./error.log', $value['name'].$value['url']."页面发生变化,需要填写验证码\r\n",FILE_APPEND);
    //             die;
    //         }
    //         $pattern_parent = "/id=\"region-nav-sub\"[\s\S]+<\/div>/U";
    //         preg_match_all($pattern_parent, $content_parent , $parent);  //获得二级地区的父元素
    //         unset($content_parent);
    //         $pattern = "/(?P<citylist>www.+)\" data-cat.+<span>(?P<name>[\x80-\xff]+)<\/span>/";
    //         $content = $parent[0][0];
    //         preg_match_all($pattern, $content , $result);  //获得二级地区url
    //         unset($content);
    //         $list = array();
    //         foreach ($result['citylist'] as $k => $v) {
    //             $list[$k]['url'] = $v;
    //             $list[$k]['name'] = $result['name'][$k];
    //             $list[$k]['second_citylist_id'] = $value['id'];
    //         }
    //         if(!empty($list[0]['name'])) {
    //             if(M('third_citylist')->Addall($list)){
    //                 M('second_citylist')->where('id='.$value['id'])->setField('is_pull',1);
    //                 unset($list);
    //                 echo $value['name'].'地区抓取成功<br>';
    //             }else{
    //                 echo $value['name'].$value['url'].'地区抓取失败<br>';
    //             }
    //         }else{
    //             echo '获取的信息如下<br>';
    //             M('second_citylist')->where('id='.$value['id'])->setField('is_pull',5);
    //             file_put_contents('./error.log', $value['name'].$value['url']."没有子地区\r\n",FILE_APPEND);
    //             echo $value['name'].'没有子地区<br>';
    //         }
    //         sleep(0.15);
    //     }
    //     $this->success('完成任务');
    // }
    /**
     * 获得店铺信息 
     * @return [type] [description]
     */
    public function getUrlList() {
        $cityList = M('thirdCitylist')->where('is_pull = 0')->select();
        $cate_list = M('category')->select();
        foreach ($cityList as $key => &$value) {
            if (strpos($value['url'], "www.dianping.com") !== false) {
                $value['url'] = substr($value['url'], 2);
            } else {
                $value['url'] = "www.dianping.com" . $value['url'];
            }
            $value['url'] = "http://" .$value['url'];
            foreach ($cate_list as  $v) {
                $url = str_replace('ch10', $v['alias'], $value['url']);
                $num = $this->getNumFromUrl($url);
                for ($i=1; $i <=$num ; $i++) { 
                    $data[$i]['url'] = $url."p".$i;
                    $data[$i]['cate_id'] = $v['id'];
                    $data[$i]['third_citylist_id'] = $value['id'];
                }
                $result = M('shopListUrl')->data($data)->Addall();
                if(!$result) {
                    echo $value['name'].$value['id'].$v['alias'].'添加网址失败<br>';
                    file_put_contents('./error.log', $value['name'].$value['id'].$v['alias']."添加网址失败\r\n",FILE_APPEND);
                }
            }
            die;
        }
        
    }
    public function getNumFromUrl($url){
        return 5;
        $content = pull_url($url);
    }





















    public function pull_url($url,$is_proxy=0){
        $snoopy = new \Think\Snoopy;
        // $snoopy->agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";
        $snoopy->agent = "Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) App leWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53";
        $snoopy->maxredirs = 2;
        $snoopy->offsiteok = false;
        $snoopy->expandlinks = false;
        if($is_proxy) {
            $result = false;
            while (!$result) {
                if(!isset($ips) || count($ips)<1 ) {   //代理数组
                    $ips = $this->ips();
                }
                foreach ($ips as $key => $value) {
                    $snoopy->proxy_host = $value[0];
                    $snoopy->proxy_port = $value[1]; //使用代理
                    $result = $snoopy->fetch($url);
                    if($result) {
                        $content = $snoopy->results;
                        if(!preg_match("/抱歉！页面无法访问/", $content)){
                            echo $value[0].'可以访问<br>';
                            break;      //成功后退出
                        }
                    }else{
                        echo "呗屏蔽ip $value[0]";
                    }
                    unset($ips[$key]); //删除不能用的代理
                }
            }        
        }else{ //不使用代理
            if ($snoopy->fetch($url)) {
                if(preg_match("/抱歉！页面无法访问/", $content)){
                    echo $value[0].'可以访问<br>';
                    die;      //成功后退出
                }
                return $content = $snoopy->results;
            }else{
                return false;
            }
        }
    }

    /**
     * 返回代理ip的数组
     */
    public function ips(){
        $str = file_get_contents('http://dps.kdlapi.com/api/getdps/?orderid=923085920779185&num=100&pt=1&ut=1&sep=1');
        file_put_contents('./ips.txt', $str);
        $str = explode("\r\n",$str);
        foreach ($str as $key => &$value) {
            $value = explode(':', $value);
        }
        return $str;
    }
}


/**
 * currentDate:2018-07-08 15:31:49
 * userIp:123.119.4.60, 10.72.40.8
 * userAgent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0
 */